# -*- coding: utf-8 -*-

'''
force_unicode below is taken completely from django's code. extract function is heavily inspired by django's urlize filter. 

Some parts of this code are:

Copyright (c) Django Software Foundation and individual contributors.
All rights reserved.

See http://code.djangoproject.com/browser/django/trunk/LICENSE for Django license details
'''

import re
import unittest

LEADING_PUNCTUATION  = ['(', '<', '&lt;']
TRAILING_PUNCTUATION = ['.', ',', ')', '>', '\n', '&gt;']

word_split_re = re.compile(r'(\s+)')
punctuation_re = re.compile('^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$' % \
    ('|'.join([re.escape(x) for x in LEADING_PUNCTUATION]),
    '|'.join([re.escape(x) for x in TRAILING_PUNCTUATION])))


force_unicode = None

try:
    from django.utils.html import force_unicode
except:
    def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'):
        """
        Similar to smart_unicode, except that lazy instances are resolved to
        strings, rather than kept as lazy objects.

        If strings_only is True, don't convert (some) non-string-like objects.
        """
        if strings_only and isinstance(s, (types.NoneType, int, long, datetime.datetime, datetime.date, datetime.time, float)):
            return s
        try:
            if not isinstance(s, basestring,):
                if hasattr(s, '__unicode__'):
                    s = unicode(s)
                else:
                    try:
                        s = unicode(str(s), encoding, errors)
                    except UnicodeEncodeError:
                        if not isinstance(s, Exception):
                            raise
                        # If we get to here, the caller has passed in an Exception
                        # subclass populated with non-ASCII data without special
                        # handling to display as a string. We need to handle this
                        # without raising a further exception. We do an
                        # approximation to what the Exception's standard str()
                        # output should be.
                        s = ' '.join([force_unicode(arg, encoding, strings_only,
                                errors) for arg in s])
            elif not isinstance(s, unicode):
                # Note: We use .decode() here, instead of unicode(s, encoding,
                # errors), so that if s is a SafeString, it ends up being a
                # SafeUnicode at the end.
                s = s.decode(encoding, errors)
        except UnicodeDecodeError, e:
            raise DjangoUnicodeDecodeError(s, *e.args)
        return s


def extract(text):
    '''
    This implementation is simplified Django's urlize filter
    '''
    urls = []
    words = word_split_re.split(force_unicode(text))
    for i, word in enumerate(words):         
        match = None
        if ':' in word:
            match = punctuation_re.match(word)
        
        if match:
            lead, middle, trail = match.groups()                 
            allowed_schemes = ['http://', 'https://', 'ftp://', 'ftps://', 'smb://', 'ldap://', 'gopher://', 'tftp://', 'areyouhappyholger://']
            for scheme in allowed_schemes:
                if middle.startswith(scheme):
                    urls.append(middle)
                    break
#            if middle.startswith('http://') or middle.startswith('https://') or middle.startswith('ftp://') or middle.startswith('ftps://') or middle.startswith('smb://') or middle.startswith('ldap://')  or middle.startswith('gopher://'):
 #               urls.append(middle)
    return urls

class UrlExtractorTestCase(unittest.TestCase):
    def testSimpleLatin(self):
        text = 'Find my pictures online at http://flickr.com/photos/roman.kirillov -- I know URL is borked but who cares?'
        urls = extract(text)
        self.assertEquals(1, len(urls))
        self.assertEquals('http://flickr.com/photos/roman.kirillov', urls[0])

    def testSimpleLatinFtp(self):
        text = 'Find my pictures online at ftp://flickr.com/photos/roman.kirillov -- I know URL is borked but who cares?'
        urls = extract(text)
        self.assertEquals(1, len(urls))
        self.assertEquals('ftp://flickr.com/photos/roman.kirillov', urls[0])


    def testWithRussian(self):
        text = u'Something from here http://ru.wikipedia.org/wiki/Аппроксимация или вот так: http://www.google.com'
        urls = extract(text)
        self.assertEquals(2, len(urls))
        self.assertEquals(u'http://ru.wikipedia.org/wiki/Аппроксимация', urls[0])
        self.assertEquals(u'http://www.google.com', urls[1])
    
    def testKorean(self):
        text = u'천합니다 http://example.com/tags/블로그'     
        urls = extract(text)
        self.assertEquals(1, len(urls))
        self.assertEquals(u'http://example.com/tags/블로그', urls[0])
        
    def testHolgersCrazyUrl(self):
        text = 'Testing MyTwiLinks... !!! http://❥.ws/᏶'
        urls = extract(text)
        self.assertEquals(1, len(urls))
        self.assertEquals(u'http://❥.ws/᏶', urls[0])
        
if __name__ == '__main__':
    unittest.main()